'''
Created on Apr 19, 2012

@author: samindaw
'''
from xgoogle.search import GoogleSearch
import re
import urllib2

class SearchCorpus:
    def get_occurances(self, wordpair):
        pass
    def get_search_data(self,words):
        pass
    def search_pairs(self, wordpair):
        occurances=self.get_occurances(wordpair)
        pairs=[]
        [pairs.append((occurance.split()[0].lower(),occurance.split()[-1].lower(),len(occurance.split())-2)) for occurance in occurances]
        pair_set=set(pairs)
        result=[]
        [result.append((pair,pairs.count(pair)))for pair in pair_set]
        return result;
    def search_sentences(self, words,must_contain_all=False):
        results=self.get_search_data(words,must_contain_all)
        frequencies={}
        for w in words:
            [frequencies.setdefault(w,[]).extend(re.findall(r""+w+"",result.data.lower())) for result in results]
        return frequencies
class GoogleSearchCorpus(SearchCorpus):
    def __init__(self):
        self.gs=GoogleSearch(None)

    def get_data_from_url(self,url):
        website = urllib2.urlopen(url)
        return website.read()     
        
    def get_occurances(self, wordpair):
        w1=wordpair[0].lower()
        w2=wordpair[1].lower()
        self.gs.query=w1+" * "+w2
        allresults=[]
        results=self.gs.get_results()
        while (len(results)>0):
            allresults.extend(results)
            results=self.gs.get_results()
        phrases=[]
        [phrases.extend(re.findall(r""+w1+" .*? "+w2+"",result.desc.encode('utf8').lower())) for result in allresults]
        return phrases
    
    def get_search_data(self,words,must_contain_all=False):
        prefix="+" if must_contain_all else ""
        search_query=" ".join([prefix+'"'+w+'"' for w in words])
        self.gs.query=search_query
        allresults=[]
        results=self.gs.get_results()
#        while (len(results)>0):
#            allresults.extend(results)
        allresults.append(results[0])
#        for result in allresults:
#            data=self.get_data_from_url(result.url)
#            result.data=data
        allresults[0].data=self.get_data_from_url(allresults[0].url)
        return allresults